# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import requests
from concurrent.futures import ThreadPoolExecutor
import time
import os


class WeibovideoPipeline:
    pools = ThreadPoolExecutor(max_workers=8)

    def process_item(self, item, spider):
        title = item['title'].split('.')[0]
        args = [item['url'], title]
        self.pools.submit(lambda p: self.download(*p), args)
        return item

    #  下载视频
    def download(self, url, title):
        #  获取当前的根目录
        path = os.getcwd()
        #  判断vlog目录是否存在　, 没有创建
        if not os.path.exists(path + '/vlog'):
            os.mkdir(path + '/vlog')
        headers = {
            'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0'
        }
        start = time.time()  # 下载开始时间
        response = requests.get(url=url, headers=headers, stream=True)  # stream=True必须写上
        size = 0  # 初始化已下载大小
        chunk_size = 1024  # 每次下载的数据大小
        content_size = int(response.headers['content-length'])  # 下载文件总大小
        try:
            if response.status_code == 200:  # 判断是否响应成功
                print('\n[文件大小]:{size:.2f} MB'.format(
                    size=content_size / chunk_size / 1024))  # 开始下载，显示下载文件大小
                with open(f'{path}/vlog/{title}.mp4', 'ab') as file:  # 显示进度条
                    for data in response.iter_content(chunk_size=chunk_size):
                        file.write(data)
                        size += len(data)
                        print('\r' + '[下载进度]:%s%.2f%%' % (
                            '#' * int(size * 50 / content_size), float(size / content_size * 100)), end=' ')
                        file.flush()
            end = time.time()  # 下载结束时间
            print('下载完成 !用时: %.2f秒' % (end - start))  # 输出下载用时时间
        except Exception as e:
            print(e)
